from twisted.python.threadpool import ThreadPool
import os, time, urllib

tp = ThreadPool(1,10)

class progressBar:
    def __init__(self, minValue = 0, maxValue = 10, totalWidth=12):
        self.progBar = "[]"   # This holds the progress bar string
        self.min = minValue
        self.max = maxValue
        self.span = maxValue - minValue
        self.width = totalWidth
        self.amount = 0       # When amount == max, we are 100% done 
        self.updateAmount(0)  # Build progress bar string

    def updateAmount(self, newAmount = 0):
        if newAmount < self.min: newAmount = self.min
        if newAmount > self.max: newAmount = self.max
        self.amount = newAmount

        # Figure out the new percent done, round to an integer
        diffFromMin = float(self.amount - self.min)
        percentDone = (diffFromMin / float(self.span)) * 100.0
        percentDone = round(percentDone)
        percentDone = int(percentDone)

        # Figure out how many hash bars the percentage should be
        allFull = self.width - 2
        numHashes = (percentDone / 100.0) * allFull
        numHashes = int(round(numHashes))

        # build a progress bar with hashes and spaces
        self.progBar = "[" + '#'*numHashes + ' '*(allFull-numHashes) + "]"

        # figure out where to put the percentage, roughly centered
        percentPlace = (len(self.progBar) / 2) - len(str(percentDone)) 
        percentString = str(percentDone) + "%"

        # slice the percentage into the bar
        #self.progBar = self.progBar[0:percentPlace] + percentString +self.progBar[percentPlace+len(percentString):]
        self.progBar = self.progBar[0:percentPlace] + self.progBar[percentPlace+len(percentString):] + percentString

    def __str__(self):
        return str(self.progBar)

def myReportHook(count, blockSize, totalSize):
    import sys
    global prog

    if not prog:
            prog = progressBar(0,totalSize,50)
    #print count, blockSize, totalSize
    #prog = progressBar(0, totalSize, 77)
    prog.updateAmount(count*blockSize)
    sys.stdout.write (str(prog))
    sys.stdout.write ("\r")
    #print count * (blockSize/1024) , "kb of " , (totalSize/1024) , "kb

def getfile(url, pardir=None, index=None):
    print 'start to get:%s'%url
    filename = os.path.split(url)[-1]
    for c in '\/:*?"<>|':
        filename = filename.replace(c,'%02X'%ord(c))
    if not filename.lower().endswith(".jpg"):
        filename+='.jpg'
    if pardir:
        finalfilename = os.path.join(pardir, filename)
    if os.path.exists(finalfilename) and os.path.getsize(finalfilename)>0:
        print 'already downloaded'
        newname = os.path.join(pardir, '%06d__%s'%(index, filename))
        os.rename(finalfilename, newname)
        print 'rename to %s'%newname
        return
    finalfilename = os.path.join(pardir, '%06d__%s'%(index, filename))
    if os.path.exists(finalfilename) and os.path.getsize(finalfilename)>0:
        return
    urllib.urlretrieve(url,finalfilename, reporthook=myReportHook)
    print '%s done, save to %s'%(url,finalfilename)

def cbRunDone(result):
    global tp,allended
    if allended:
        return
##    print 'queue:%s\nwaiters:%s\nworkers:%s\ntotal:%s\n'%(len(tp.q.queue),
##                                                          len(tp.waiters),
##                                                          len(tp.working),
##                                                          len(tp.threads))
    print '%s jobs remain'%len(tp.q.queue)
    if len(tp.q.queue)==0:
        allended = True
        try:
            tp.stop()
        except:
            pass
        finally:
            print 'ended in cbRunDone'
flist = open('foundimg', 'r')
jobs = flist.readlines()
flist.close()

dirname = "".join(['%02d'%x for x in time.localtime()[:3]])

os.popen('mkdir %s'%dirname)

prog = None
allended = False
print '%s lines, start from'%len(jobs),
stnum = raw_input().strip()
try:
    stnum = int(stnum)
except:
    stnum=0
for index, job in enumerate(jobs):
    if index < stnum:
        continue
    lineitems = job.split()
    tp.dispatchWithCallback(None, cbRunDone, cbRunDone, getfile, lineitems[0], dirname, index)

tp.start()
